string = "to test [pemit(%#,blah [ufun(#4,foo)] [add(1,0)] blah)] blah"
require 'strscan'
class Tokenizer
        def Tokenizer.tokenize str, state = :function
          s = StringScanner.new str
          eval_deepth = 0
          function_deepth = 0 
          tokens = []
          lambda_deepth = 0
          lambdas = []
          while not s.eos?
            if s.scan(/%(q[\da-z]|[vwx][a-z]|.)/i)
              tokens << [s.matched, :substitution]
            elsif s.scan(/\[/)
              state = :function
              eval_deepth += 1
              tokens << [s.matched, :eval_start]
            elsif (eval_deepth > 0) and s.scan(/\]/)
              eval_deepth -= 1
              state = :string
              tokens << [s.matched, :eval_end]
            elsif (function_deepth > 0) and s.scan(/,/)
              if lambdas.last
                lambda_deepth -= 1
                lambdas.pop
                tokens << [s.matched, :lambda_end]
              end
              tokens << [s.matched, :function_param_sep]
              state = :function
            else
              case state
                when :function
                if s.scan(/[a-zA-Z][\w_]*\(/)
                  tokens << [s.matched, :function_start]
                  function_deepth += 1
                  lambdas << false
                elsif (function_deepth > 0) and s.scan(/\)/)
                  function_deepth -= 1
                  state = :string
                  if lambdas.last
                    lambda_deepth -= 1
                    lambdas.pop
                    tokens << [s.matched, :lambda_end]
                  end
                  lambdas.pop
                  tokens << [s.matched, :function_end]
                elsif (function_deepth > 0) and s.scan(/#lambda\//)
                  tokens << [s.matched, :lambda]
                  lambdas << true
                  lambda_deepth += 1
                else
                  state = :string
                end
                when :string
                if  ((function_deepth == 0) and s.scan(/(\\.|[^%\[\]])+/))
                  tokens << [s.matched, :string]
                elsif  ((function_deepth > 0) and s.scan(/(\\.|[^%,\[\)\]])+/))
                  tokens << [s.matched, :string]
                  state = :function
                elsif (function_deepth > 0) and s.scan(/\)/)
                  function_deepth -= 1
                  state = :string
                  if lambdas.last
                    lambda_deepth -= 1
                    lambdas.pop
                    tokens << [s.matched, :lambda_end]
                  end
                  lambdas.pop
                  tokens << [s.matched, :function_end]
                else
                  raise RVM::Classes[:error].new(3, "Tokenizer error at #{s.pos} in '#{str}'")
                end
              end
            end
          end
          tokens
        end
      end
def colourCode(string,wi,dformat=1)
  wi.value = ""
  tokens = Tokenizer.tokenize string
  ident = 0
  tokens.each {|t|
p "======\n========"*12
    case t[1]
      when :function_end
       fclr = Wx::RED
       ident = ident - 5
       t[0]="\n"+" "*ident+t[0] if dformat == 1
      when :function_start
       fclr =  Wx::RED
       ident = ident + 5
       t[0]="\n"+" "*ident+t[0] if dformat == 1
      when :eval_start
        fclr = Wx::YELLOW
      when :eval_end
       fclr = Wx::YELLOW
      when :substitution
        fclr = Wx::BLUE
      when :function_param_sep
        fclr = Wx::BLUE
    else
      fclr = Wx::BLACK
    end
    f = Wx::Font.new(10, Wx::MODERN, -1,-1, false)
    wi.set_default_style Wx::TextAttr::new(fclr,Wx::WHITE,f)
    wi.append_text(t[0])
  }
end

def format string
  tokens = Tokenizer.tokenize string
  str1 = ""
  ident = 0
  tokens.each {|t|
    case t[1]
      when :function_end
        ident = ident - 5
        str1=str1+"\n"+" "*ident+t[0]
      when :function_start
        ident = ident + 5
        str1=str1+"\n"+" "*ident+t[0]
#     when :eval_end
#       str1=str1+t[0]+"\n"+" "*ident
    else
      str1=str1+t[0]
    end
  }
  str1
end
def unformat string
  str = ""
  string.each_line("\n") {|line|
    case line
      when /^\#/
    else
      str=str+line.gsub("\n",'').strip
    end
  }
  str
end
s = format string
puts s
puts unformat s